{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The fisrt stage of training \n",
    "The initial training on pseudo-labeled data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset, load_metric, Dataset, DatasetDict, load_from_disk\n",
    "from huggingface_hub import notebook_login\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 331,
     "referenced_widgets": [
      "09aec9635cf641bdb821c81a2bbdbc9f",
      "571eb22bad354e94b3fba1cac59e6608",
      "c3ad5d149e0047628c3e0aefd29c7460",
      "2b6e7e05ba0a4419aa8fdea84fcc2354",
      "2bdecb5267004d1fb534cfd5a3fea811",
      "d5b3dc1fa6c743e7b1e05a091c22b270",
      "a667e03927f748ba9db38ddd77ed17b8",
      "318c86b9f6eb4266855180543d70399f",
      "5abe0708ef614c7caa0f6faa10490c4b",
      "5a0871be46944d6d857bf3da82bd2f85",
      "6ead92dda89645149a5e0a7e4060bcb7",
      "af2a6221808f40b4813a158e3ae91559",
      "8a0a661799814f809a84131c767dfda2",
      "40b4b89c55894cad9722d9c52a2af235",
      "ed568e897d1c4b7a9eb2c9e69997d29b",
      "aa0798372ed149a18994104d7f9a3d97",
      "3ab74c3f04634c16b24630f9b6d1798c"
     ]
    },
    "id": "e-wI3YzurfCD",
    "outputId": "5c012504-773c-44b2-9f4e-a55079ef4b26"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b87760e03d294e6f86cfe334cb577324",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "nHXDl8ifroF8"
   },
   "outputs": [],
   "source": [
    "model_checkpoint = \"microsoft/deberta-v3-base\"\n",
    "batch_size = 24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "IreSlFmlIrIm"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/monty/projects/pii-ner/utils/misc.py:38: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
      "  _seqeval_metric = load_metric(\"seqeval\")\n",
      "Using the latest cached version of the module from /home/monty/.cache/huggingface/modules/datasets_modules/metrics/seqeval/c8563af43bdce095d0f9e8b8b79c9c96d5ea5499b3bf66f90301c9cb82910f11 (last modified on Thu Feb 16 17:58:29 2023) since it couldn't be found locally at seqeval, or remotely on the Hugging Face Hub.\n",
      "Some weights of the model checkpoint at microsoft/deberta-v3-base were not used when initializing DebertaV2ForTokenClassification: ['deberta.embeddings.position_embeddings.weight', 'lm_predictions.lm_head.LayerNorm.weight', 'mask_predictions.LayerNorm.weight', 'mask_predictions.LayerNorm.bias', 'mask_predictions.classifier.bias', 'lm_predictions.lm_head.bias', 'mask_predictions.classifier.weight', 'mask_predictions.dense.weight', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.dense.weight', 'mask_predictions.dense.bias', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaV2ForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2ForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DebertaV2ForTokenClassification were not initialized from the model checkpoint at microsoft/deberta-v3-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/transformers/convert_slow_tokenizer.py:434: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  warnings.warn(\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer\n",
    "from transformers import DebertaV2TokenizerFast\n",
    "\n",
    "from utils import LABEL2ID, ID2LABEL\n",
    "\n",
    "\n",
    "model = AutoModelForTokenClassification.from_pretrained(model_checkpoint, num_labels=len(ID2LABEL))\n",
    "tokenizer = DebertaV2TokenizerFast.from_pretrained(model_checkpoint, add_prefix_space=True)\n",
    "\n",
    "model.config.id2label = {str(i):label for i, label in enumerate(ID2LABEL)}\n",
    "model.config.label2id = LABEL2ID\n",
    "\n",
    "tokenizer.model_max_length = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105,
     "referenced_widgets": [
      "40e9832b79644dfbbcd6cd6ea8bfc8c7",
      "9c872cf485ad4ae8b8359c33f7c3ea65",
      "11c41adf6b904d10b0768935709c68d1",
      "9491e625e9604427a640b5aa11e67745",
      "49118a87b2204853bd81060094788a8a",
      "72d64b7afb5147c5aad0416674520849",
      "173b4df1dad44cec83270befa7392b1e",
      "55253a79d100489c8512ff7e7f11f92c",
      "f919b355bcd344059b9161e6fac7cbac",
      "e0c423cb57e949f08255453100fd3e69",
      "3bf5b65c1c9a4ce0be50e3e2fce60c87"
     ]
    },
    "id": "s_AY1ATSIrIq",
    "outputId": "db669025-726d-4e81-e7eb-7133640aacfd"
   },
   "outputs": [],
   "source": [
    "train_dataset = load_dataset(\"bigcode/pseudo-labeled-python-data-pii-detection-filtered\", use_auth_token=True)['train']\n",
    "dev_dataset = load_dataset(\"bigcode/pii-for-code-v2/\", use_auth_token=True)['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f48f92d333344639887a1b3d2ce90a47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/400 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e72a8f2e5a14bccb35194be4a982307",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/400 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (8742 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['content', 'language', 'license', 'path', 'annotation_id', 'pii', 'id', 'fold', 'input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping', 'labels'],\n",
       "    num_rows: 400\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utils import label_tokenized\n",
    "\n",
    "def tokenize_and_label(entry, tokenizer=tokenizer):\n",
    "    inputs = tokenizer.encode_plus(entry['content'], return_offsets_mapping=True, add_special_tokens=False)\n",
    "    entry.update(inputs)\n",
    "    return label_tokenized(entry)\n",
    "\n",
    "dev_dataset = dev_dataset.map(lambda x: dict(pii=json.loads(x['pii'])))\n",
    "dev_dataset = dev_dataset.map(tokenize_and_label)\n",
    "dev_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4be5f3c834094bb69f39b8806d754e66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17678 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "faafc5543a974652844b70efbd032425",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#0:   0%|          | 0/2210 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65b60ce5bb2942d9863ff0855e01d976",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#1:   0%|          | 0/2210 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca0fbad9cfa24d3997d329ab7c4865b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#2:   0%|          | 0/2210 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d7bb3ad97194737a7f1e405017583d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#3:   0%|          | 0/2210 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1602d1c85e746149c39c33e976e7c7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#4:   0%|          | 0/2210 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed8f0f2415e34f49ae91aa2ce84242c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#5:   0%|          | 0/2210 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a81dd1bc9fb46c2976c43569c04eefc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#6:   0%|          | 0/2209 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7eb06f92a4f479db6a3c12fd1d0d779",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#7:   0%|          | 0/2209 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "TypeError",
     "evalue": "chunk_dataset() got an unexpected keyword argument 'load_from_cache_file'",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "Input \u001B[0;32mIn [12]\u001B[0m, in \u001B[0;36m<cell line: 5>\u001B[0;34m()\u001B[0m\n\u001B[1;32m      3\u001B[0m train_dataset \u001B[38;5;241m=\u001B[39m train_dataset\u001B[38;5;241m.\u001B[39mmap(\u001B[38;5;28;01mlambda\u001B[39;00m x: \u001B[38;5;28mdict\u001B[39m(pii\u001B[38;5;241m=\u001B[39mjson\u001B[38;5;241m.\u001B[39mloads(x[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mpii\u001B[39m\u001B[38;5;124m'\u001B[39m])), load_from_cache_file \u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m)\n\u001B[1;32m      4\u001B[0m train_dataset \u001B[38;5;241m=\u001B[39m train_dataset\u001B[38;5;241m.\u001B[39mmap(tokenize_and_label, num_proc\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m8\u001B[39m, load_from_cache_file \u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m)\n\u001B[0;32m----> 5\u001B[0m train_dataset \u001B[38;5;241m=\u001B[39m \u001B[43mchunk_dataset\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtrain_dataset\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtokenizer\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mload_from_cache_file\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m)\u001B[49m\n",
      "\u001B[0;31mTypeError\u001B[0m: chunk_dataset() got an unexpected keyword argument 'load_from_cache_file'"
     ]
    }
   ],
   "source": [
    "from utils import chunk_dataset\n",
    "\n",
    "train_dataset = train_dataset.map(lambda x: dict(pii=json.loads(x['pii'])))\n",
    "train_dataset = train_dataset.map(tokenize_and_label, num_proc=8)\n",
    "train_dataset = chunk_dataset(train_dataset, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask', 'labels', 'id', 'chunk_id'],\n",
       "    num_rows: 121080\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f536bbcf9d3f4e808068af308c6254ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/400 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa1d42edfde34c76a7089b7d4fd012a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/400 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'attention_mask', 'labels', 'id', 'chunk_id'],\n",
       "        num_rows: 121080\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['input_ids', 'attention_mask', 'labels', 'id', 'chunk_id'],\n",
       "        num_rows: 2040\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'attention_mask', 'labels', 'id', 'chunk_id'],\n",
       "        num_rows: 3853\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_dataset = DatasetDict(\n",
    "    train = train_dataset,\n",
    "    validation = chunk_dataset(dev_dataset, tokenizer),\n",
    "    test = chunk_dataset(dev_dataset, tokenizer, overlap_freq=2),\n",
    ")\n",
    "ner_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForTokenClassification, EarlyStoppingCallback\n",
    "\n",
    "data_collator = DataCollatorForTokenClassification(tokenizer)\n",
    "model_name = model_checkpoint.split(\"/\")[-1]\n",
    "args = TrainingArguments(\n",
    "    f\"{model_name}-pretrained\",\n",
    "    overwrite_output_dir=True,\n",
    "    evaluation_strategy = \"steps\",\n",
    "    save_strategy='steps',\n",
    "    num_train_epochs=1,\n",
    "    eval_steps=300,\n",
    "    save_steps=300,\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    metric_for_best_model=\"f1\",\n",
    "    load_best_model_at_end = True,\n",
    "    weight_decay=0.01,\n",
    "    logging_steps=10,\n",
    "    save_total_limit=30,\n",
    "    push_to_hub=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "imY1oC3SIrJf"
   },
   "outputs": [],
   "source": [
    "from utils import compute_metrics\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=ner_dataset[\"train\"],\n",
    "    eval_dataset=ner_dataset[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics,\n",
    "    callbacks=[EarlyStoppingCallback(early_stopping_patience = 30, early_stopping_threshold= 1e-3)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 121080\n",
      "  Num Epochs = 1\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 7568\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7568' max='7568' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7568/7568 1:58:39, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Avg.precision</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>Ambiguous</th>\n",
       "      <th>Email</th>\n",
       "      <th>Ip Address</th>\n",
       "      <th>Key</th>\n",
       "      <th>Name</th>\n",
       "      <th>Password</th>\n",
       "      <th>Username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.049900</td>\n",
       "      <td>0.024034</td>\n",
       "      <td>0.649894</td>\n",
       "      <td>0.302181</td>\n",
       "      <td>0.400413</td>\n",
       "      <td>0.344430</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.612245</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.066100</td>\n",
       "      <td>0.025051</td>\n",
       "      <td>0.727751</td>\n",
       "      <td>0.434845</td>\n",
       "      <td>0.623323</td>\n",
       "      <td>0.512299</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.933638</td>\n",
       "      <td>0.423077</td>\n",
       "      <td>0.052083</td>\n",
       "      <td>0.758221</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.402277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>0.041700</td>\n",
       "      <td>0.054753</td>\n",
       "      <td>0.607098</td>\n",
       "      <td>0.200584</td>\n",
       "      <td>0.637771</td>\n",
       "      <td>0.305185</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.314797</td>\n",
       "      <td>0.351779</td>\n",
       "      <td>0.035714</td>\n",
       "      <td>0.771481</td>\n",
       "      <td>0.116129</td>\n",
       "      <td>0.402662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.021500</td>\n",
       "      <td>0.019985</td>\n",
       "      <td>0.792680</td>\n",
       "      <td>0.528986</td>\n",
       "      <td>0.678019</td>\n",
       "      <td>0.594301</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.795322</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.184211</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.543689</td>\n",
       "      <td>0.394973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.023700</td>\n",
       "      <td>0.040654</td>\n",
       "      <td>0.711795</td>\n",
       "      <td>0.467731</td>\n",
       "      <td>0.695562</td>\n",
       "      <td>0.559336</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.799220</td>\n",
       "      <td>0.553846</td>\n",
       "      <td>0.104784</td>\n",
       "      <td>0.761726</td>\n",
       "      <td>0.562500</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.017200</td>\n",
       "      <td>0.029019</td>\n",
       "      <td>0.695333</td>\n",
       "      <td>0.376392</td>\n",
       "      <td>0.697626</td>\n",
       "      <td>0.488969</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.811133</td>\n",
       "      <td>0.586572</td>\n",
       "      <td>0.032138</td>\n",
       "      <td>0.780399</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.503093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>0.049400</td>\n",
       "      <td>0.037411</td>\n",
       "      <td>0.718493</td>\n",
       "      <td>0.468475</td>\n",
       "      <td>0.713106</td>\n",
       "      <td>0.565466</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.771536</td>\n",
       "      <td>0.684615</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.760417</td>\n",
       "      <td>0.580153</td>\n",
       "      <td>0.451730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>0.017000</td>\n",
       "      <td>0.032984</td>\n",
       "      <td>0.776229</td>\n",
       "      <td>0.426748</td>\n",
       "      <td>0.724458</td>\n",
       "      <td>0.537108</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.850716</td>\n",
       "      <td>0.395745</td>\n",
       "      <td>0.067901</td>\n",
       "      <td>0.798493</td>\n",
       "      <td>0.537143</td>\n",
       "      <td>0.421900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>0.034800</td>\n",
       "      <td>0.034663</td>\n",
       "      <td>0.728199</td>\n",
       "      <td>0.515337</td>\n",
       "      <td>0.693498</td>\n",
       "      <td>0.591289</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.819802</td>\n",
       "      <td>0.676923</td>\n",
       "      <td>0.114286</td>\n",
       "      <td>0.767790</td>\n",
       "      <td>0.589928</td>\n",
       "      <td>0.417423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.030500</td>\n",
       "      <td>0.038710</td>\n",
       "      <td>0.673239</td>\n",
       "      <td>0.553344</td>\n",
       "      <td>0.717234</td>\n",
       "      <td>0.624719</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.804642</td>\n",
       "      <td>0.752137</td>\n",
       "      <td>0.092527</td>\n",
       "      <td>0.800731</td>\n",
       "      <td>0.579310</td>\n",
       "      <td>0.503018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3300</td>\n",
       "      <td>0.020800</td>\n",
       "      <td>0.034551</td>\n",
       "      <td>0.738151</td>\n",
       "      <td>0.347558</td>\n",
       "      <td>0.697626</td>\n",
       "      <td>0.463967</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.574586</td>\n",
       "      <td>0.706383</td>\n",
       "      <td>0.035928</td>\n",
       "      <td>0.788104</td>\n",
       "      <td>0.407895</td>\n",
       "      <td>0.438449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3600</td>\n",
       "      <td>0.034800</td>\n",
       "      <td>0.019907</td>\n",
       "      <td>0.776724</td>\n",
       "      <td>0.503623</td>\n",
       "      <td>0.717234</td>\n",
       "      <td>0.591741</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.737030</td>\n",
       "      <td>0.756303</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.814259</td>\n",
       "      <td>0.464000</td>\n",
       "      <td>0.459459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3900</td>\n",
       "      <td>0.016200</td>\n",
       "      <td>0.038286</td>\n",
       "      <td>0.706181</td>\n",
       "      <td>0.394573</td>\n",
       "      <td>0.720330</td>\n",
       "      <td>0.509861</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.739677</td>\n",
       "      <td>0.652174</td>\n",
       "      <td>0.045524</td>\n",
       "      <td>0.810707</td>\n",
       "      <td>0.607407</td>\n",
       "      <td>0.458904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4200</td>\n",
       "      <td>0.019000</td>\n",
       "      <td>0.027560</td>\n",
       "      <td>0.761597</td>\n",
       "      <td>0.387541</td>\n",
       "      <td>0.725490</td>\n",
       "      <td>0.505210</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.710120</td>\n",
       "      <td>0.725806</td>\n",
       "      <td>0.043716</td>\n",
       "      <td>0.796429</td>\n",
       "      <td>0.526316</td>\n",
       "      <td>0.504780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>0.020100</td>\n",
       "      <td>0.031877</td>\n",
       "      <td>0.763226</td>\n",
       "      <td>0.384743</td>\n",
       "      <td>0.718266</td>\n",
       "      <td>0.501080</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.750455</td>\n",
       "      <td>0.530973</td>\n",
       "      <td>0.035714</td>\n",
       "      <td>0.809886</td>\n",
       "      <td>0.567164</td>\n",
       "      <td>0.494585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4800</td>\n",
       "      <td>0.019500</td>\n",
       "      <td>0.033561</td>\n",
       "      <td>0.734134</td>\n",
       "      <td>0.384946</td>\n",
       "      <td>0.738906</td>\n",
       "      <td>0.506186</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.749550</td>\n",
       "      <td>0.664207</td>\n",
       "      <td>0.044855</td>\n",
       "      <td>0.807339</td>\n",
       "      <td>0.563758</td>\n",
       "      <td>0.508227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5100</td>\n",
       "      <td>0.012500</td>\n",
       "      <td>0.027832</td>\n",
       "      <td>0.766297</td>\n",
       "      <td>0.398300</td>\n",
       "      <td>0.725490</td>\n",
       "      <td>0.514265</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.740608</td>\n",
       "      <td>0.690196</td>\n",
       "      <td>0.037901</td>\n",
       "      <td>0.808429</td>\n",
       "      <td>0.567568</td>\n",
       "      <td>0.507143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5400</td>\n",
       "      <td>0.012700</td>\n",
       "      <td>0.038109</td>\n",
       "      <td>0.737661</td>\n",
       "      <td>0.404803</td>\n",
       "      <td>0.730650</td>\n",
       "      <td>0.520971</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.739286</td>\n",
       "      <td>0.735537</td>\n",
       "      <td>0.048641</td>\n",
       "      <td>0.833013</td>\n",
       "      <td>0.581560</td>\n",
       "      <td>0.497278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5700</td>\n",
       "      <td>0.017500</td>\n",
       "      <td>0.023654</td>\n",
       "      <td>0.785633</td>\n",
       "      <td>0.452578</td>\n",
       "      <td>0.733746</td>\n",
       "      <td>0.559843</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.780303</td>\n",
       "      <td>0.726531</td>\n",
       "      <td>0.079295</td>\n",
       "      <td>0.787546</td>\n",
       "      <td>0.565217</td>\n",
       "      <td>0.460800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>0.017500</td>\n",
       "      <td>0.032962</td>\n",
       "      <td>0.721017</td>\n",
       "      <td>0.464432</td>\n",
       "      <td>0.700722</td>\n",
       "      <td>0.558618</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.735714</td>\n",
       "      <td>0.684825</td>\n",
       "      <td>0.072562</td>\n",
       "      <td>0.809074</td>\n",
       "      <td>0.424242</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6300</td>\n",
       "      <td>0.004500</td>\n",
       "      <td>0.031914</td>\n",
       "      <td>0.757278</td>\n",
       "      <td>0.444514</td>\n",
       "      <td>0.731682</td>\n",
       "      <td>0.553042</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.745027</td>\n",
       "      <td>0.671698</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.811918</td>\n",
       "      <td>0.579710</td>\n",
       "      <td>0.490231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6600</td>\n",
       "      <td>0.030200</td>\n",
       "      <td>0.027673</td>\n",
       "      <td>0.756624</td>\n",
       "      <td>0.510576</td>\n",
       "      <td>0.722394</td>\n",
       "      <td>0.598291</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.758242</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.103976</td>\n",
       "      <td>0.823077</td>\n",
       "      <td>0.584615</td>\n",
       "      <td>0.477718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6900</td>\n",
       "      <td>0.028500</td>\n",
       "      <td>0.027783</td>\n",
       "      <td>0.761812</td>\n",
       "      <td>0.497509</td>\n",
       "      <td>0.721362</td>\n",
       "      <td>0.588880</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.743682</td>\n",
       "      <td>0.708661</td>\n",
       "      <td>0.093842</td>\n",
       "      <td>0.815238</td>\n",
       "      <td>0.564885</td>\n",
       "      <td>0.481416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7200</td>\n",
       "      <td>0.013500</td>\n",
       "      <td>0.029638</td>\n",
       "      <td>0.757564</td>\n",
       "      <td>0.438885</td>\n",
       "      <td>0.715170</td>\n",
       "      <td>0.543956</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.747731</td>\n",
       "      <td>0.714859</td>\n",
       "      <td>0.071287</td>\n",
       "      <td>0.820611</td>\n",
       "      <td>0.424242</td>\n",
       "      <td>0.469983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7500</td>\n",
       "      <td>0.010900</td>\n",
       "      <td>0.031233</td>\n",
       "      <td>0.754779</td>\n",
       "      <td>0.446213</td>\n",
       "      <td>0.723426</td>\n",
       "      <td>0.551969</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.745455</td>\n",
       "      <td>0.723577</td>\n",
       "      <td>0.071287</td>\n",
       "      <td>0.818356</td>\n",
       "      <td>0.550725</td>\n",
       "      <td>0.477352</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-300\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-300/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-300/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-300/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-300/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-600\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-600/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-600/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-600/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-600/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-900\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-900/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-900/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-900/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-900/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-1200\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1200/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1200/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1200/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1200/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-1500\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1500/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1500/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1500/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1500/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-1800\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1800/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1800/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1800/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-1800/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-2100\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2100/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2100/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2100/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2100/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-2400\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2400/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2400/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2400/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2400/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-2700\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2700/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2700/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2700/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-2700/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-3000\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3000/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3000/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3000/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3000/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-3300\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3300/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3300/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3300/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3300/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-3600\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3600/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3600/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3600/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3600/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-3900\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3900/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3900/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3900/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-3900/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-4200\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4200/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4200/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4200/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4200/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-4500\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4500/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4500/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4500/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4500/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-4800\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4800/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4800/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4800/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-4800/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-5100\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5100/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5100/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5100/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5100/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-5400\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5400/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5400/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5400/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5400/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-5700\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5700/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5700/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5700/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-5700/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-6000\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6000/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6000/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6000/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6000/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-6300\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6300/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6300/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6300/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6300/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-6600\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6600/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6600/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6600/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6600/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-6900\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6900/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6900/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6900/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-6900/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-7200\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7200/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7200/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7200/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7200/special_tokens_map.json\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: id, chunk_id. If id, chunk_id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2040\n",
      "  Batch size = 16\n",
      "/data1/monty/miniconda3/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "Saving model checkpoint to /data3/monty/deberta-v3-base-pretrained/checkpoint-7500\n",
      "Configuration saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7500/config.json\n",
      "Model weights saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7500/pytorch_model.bin\n",
      "tokenizer config file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7500/tokenizer_config.json\n",
      "Special tokens file saved in /data3/monty/deberta-v3-base-pretrained/checkpoint-7500/special_tokens_map.json\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from /data3/monty/deberta-v3-base-pretrained/checkpoint-3000 (score: 0.6247191011235955).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=7568, training_loss=0.026313426741940336, metrics={'train_runtime': 7125.7811, 'train_samples_per_second': 16.992, 'train_steps_per_second': 1.062, 'total_flos': 3.176570305184112e+16, 'train_loss': 0.026313426741940336, 'epoch': 1.0})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.chunking import compose_chunk_predictions_with_samples\n",
    "\n",
    "pred = trainer.predict(ner_dataset['test'])\n",
    "dev_dataset = compose_chunk_predictions_with_samples(dev_dataset['dev'], pred, ner_dataset['test']['id'], tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "true_labels = np.array(list(itertools.chain(*dev_dataset['labels'])))\n",
    "pred_labels = np.argmax(list(itertools.chain(*dev_dataset['pred'])), axis=-1)\n",
    "\n",
    "data = confusion_matrix(true_labels, pred_labels, labels=range(len(ID2LABEL)), normalize = 'true')\n",
    "df_cm = pd.DataFrame(data, columns=ID2LABEL, index = ID2LABEL)\n",
    "df_cm.index.name = 'Actual'\n",
    "df_cm.columns.name = 'Predicted'\n",
    "\n",
    "\n",
    "f, ax = plt.subplots(figsize=(15, 15))\n",
    "cmap = sns.cubehelix_palette(light=1, as_cmap=True)\n",
    "\n",
    "sns.heatmap(df_cm, cbar=False, annot=True, cmap=cmap, square=True, fmt='.1%',\n",
    "            annot_kws={'size': 10})\n",
    "plt.title('Actuals vs Predicted')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "piiner",
   "language": "python",
   "name": "piiner"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "008ca4ebe46d48cb989e97aab9529604": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "01b07686f6094d2391fcb9fe3cf86cc9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_71ce1ca715534844b501ad8c71f32091",
       "IPY_MODEL_da7804c13630441fbad5098822d99052",
       "IPY_MODEL_3134633945494d709b505d5cf37a99a6"
      ],
      "layout": "IPY_MODEL_4987ddd85bdd4ed9a54033b58e4b31f7"
     }
    },
    "081f5c176332457dacd0d59d7940be0d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "09aec9635cf641bdb821c81a2bbdbc9f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "VBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "VBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "VBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_571eb22bad354e94b3fba1cac59e6608",
       "IPY_MODEL_c3ad5d149e0047628c3e0aefd29c7460",
       "IPY_MODEL_2b6e7e05ba0a4419aa8fdea84fcc2354",
       "IPY_MODEL_2bdecb5267004d1fb534cfd5a3fea811",
       "IPY_MODEL_d5b3dc1fa6c743e7b1e05a091c22b270"
      ],
      "layout": "IPY_MODEL_a667e03927f748ba9db38ddd77ed17b8"
     }
    },
    "11c41adf6b904d10b0768935709c68d1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_55253a79d100489c8512ff7e7f11f92c",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f919b355bcd344059b9161e6fac7cbac",
      "value": 1
     }
    },
    "146394c42e1b4dc681bf49aa671aa3f8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_081f5c176332457dacd0d59d7940be0d",
      "placeholder": "​",
      "style": "IPY_MODEL_c42b4b6b36944f34b0cd0027c90bc968",
      "value": "Pushing dataset shards to the dataset hub: 100%"
     }
    },
    "173b4df1dad44cec83270befa7392b1e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1a5849c19ac44c5cb37de2bc04f8a8f6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "21cea385816e4179be0642f01bc58877": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2b6e7e05ba0a4419aa8fdea84fcc2354": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "CheckboxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "CheckboxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "CheckboxView",
      "description": "Add token as git credential?",
      "description_tooltip": null,
      "disabled": false,
      "indent": true,
      "layout": "IPY_MODEL_af2a6221808f40b4813a158e3ae91559",
      "style": "IPY_MODEL_8a0a661799814f809a84131c767dfda2",
      "value": true
     }
    },
    "2b92ad4cafa549a4810f212976074d25": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2bdecb5267004d1fb534cfd5a3fea811": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ButtonModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ButtonModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ButtonView",
      "button_style": "",
      "description": "Login",
      "disabled": false,
      "icon": "",
      "layout": "IPY_MODEL_40b4b89c55894cad9722d9c52a2af235",
      "style": "IPY_MODEL_ed568e897d1c4b7a9eb2c9e69997d29b",
      "tooltip": ""
     }
    },
    "3134633945494d709b505d5cf37a99a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_cc05745cc3f6421f84af9a1f492bbea2",
      "placeholder": "​",
      "style": "IPY_MODEL_a038ccac0d4748f0a4ff8dcd990bb7a3",
      "value": " 1/1 [00:00&lt;00:00, 33.98ba/s]"
     }
    },
    "318c86b9f6eb4266855180543d70399f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "39505a7df3db43a89322df20f3fe4d8d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f4bdbfe4a65c428bbcbb95a9dc4c6356",
      "placeholder": "​",
      "style": "IPY_MODEL_e8c5889da2b847f6ac1512ab06bdd281",
      "value": " 1/1 [00:06&lt;00:00,  6.12s/it]"
     }
    },
    "3ab74c3f04634c16b24630f9b6d1798c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3b736f409161470f96160bbbeca992c9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3bf5b65c1c9a4ce0be50e3e2fce60c87": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "40b4b89c55894cad9722d9c52a2af235": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "40e9832b79644dfbbcd6cd6ea8bfc8c7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_9c872cf485ad4ae8b8359c33f7c3ea65",
       "IPY_MODEL_11c41adf6b904d10b0768935709c68d1",
       "IPY_MODEL_9491e625e9604427a640b5aa11e67745"
      ],
      "layout": "IPY_MODEL_49118a87b2204853bd81060094788a8a"
     }
    },
    "4370dc13ce0a4e06b542a0b5efdbfc2c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "43d93313aaad4d7aa1ac513b8d69300c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "480990aec5504b61a2c51142b677535c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3b736f409161470f96160bbbeca992c9",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b49b0dd4ac514e6b80caa2932b5d222d",
      "value": 1
     }
    },
    "49118a87b2204853bd81060094788a8a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4987ddd85bdd4ed9a54033b58e4b31f7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4b83358fb1b64bcd9ab3b965a77fffc9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_6aa51965e96a48a4b73a0ae53b8e9b66",
       "IPY_MODEL_480990aec5504b61a2c51142b677535c",
       "IPY_MODEL_d72eb7bedd624133ac08ccb510ce901a"
      ],
      "layout": "IPY_MODEL_77a18f9c759040478fe8bc2e8abd85e3"
     }
    },
    "4d8836ba902043358883ac3729ad621a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "55253a79d100489c8512ff7e7f11f92c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "571eb22bad354e94b3fba1cac59e6608": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_318c86b9f6eb4266855180543d70399f",
      "placeholder": "​",
      "style": "IPY_MODEL_5abe0708ef614c7caa0f6faa10490c4b",
      "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
     }
    },
    "5a0871be46944d6d857bf3da82bd2f85": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5abe0708ef614c7caa0f6faa10490c4b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "652a34f18e914d56b44a52844ac56f4d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6aa51965e96a48a4b73a0ae53b8e9b66": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e7d7757a758348f898a4d308695ebed7",
      "placeholder": "​",
      "style": "IPY_MODEL_f01c225a559f443593a0989fa78fc6cb",
      "value": "Upload 1 LFS files: 100%"
     }
    },
    "6ead92dda89645149a5e0a7e4060bcb7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "71c2b430b94b4639bc8468c202801cca": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_74eb9205486944f7aa86db8bf940f992",
      "placeholder": "​",
      "style": "IPY_MODEL_4370dc13ce0a4e06b542a0b5efdbfc2c",
      "value": " 0/1 [00:00&lt;?, ?ba/s]"
     }
    },
    "71ce1ca715534844b501ad8c71f32091": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d5b2c76f54ae4bb78985ad253373408d",
      "placeholder": "​",
      "style": "IPY_MODEL_2b92ad4cafa549a4810f212976074d25",
      "value": "Creating parquet from Arrow format: 100%"
     }
    },
    "72d64b7afb5147c5aad0416674520849": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "74eb9205486944f7aa86db8bf940f992": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "74fdd4d677294768bb63225cb4684b6d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "77a18f9c759040478fe8bc2e8abd85e3": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "785502d8755f43cea3da690322f26d0d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "87a5cfe6903342b98a1ca8b8ef30f513": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_146394c42e1b4dc681bf49aa671aa3f8",
       "IPY_MODEL_ad920a4cce8a4aabbbc55308c9fd7d44",
       "IPY_MODEL_39505a7df3db43a89322df20f3fe4d8d"
      ],
      "layout": "IPY_MODEL_21cea385816e4179be0642f01bc58877"
     }
    },
    "8a0a661799814f809a84131c767dfda2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9491e625e9604427a640b5aa11e67745": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e0c423cb57e949f08255453100fd3e69",
      "placeholder": "​",
      "style": "IPY_MODEL_3bf5b65c1c9a4ce0be50e3e2fce60c87",
      "value": " 1/1 [00:00&lt;00:00, 54.85it/s]"
     }
    },
    "9c872cf485ad4ae8b8359c33f7c3ea65": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_72d64b7afb5147c5aad0416674520849",
      "placeholder": "​",
      "style": "IPY_MODEL_173b4df1dad44cec83270befa7392b1e",
      "value": "100%"
     }
    },
    "a038ccac0d4748f0a4ff8dcd990bb7a3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a1e4b90f504e4c59ac795f8a3a6553d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a667e03927f748ba9db38ddd77ed17b8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": "center",
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": "flex",
      "flex": null,
      "flex_flow": "column",
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "50%"
     }
    },
    "aa0798372ed149a18994104d7f9a3d97": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ad67045339e340a0a831369fef5bcdcf": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ad920a4cce8a4aabbbc55308c9fd7d44": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_652a34f18e914d56b44a52844ac56f4d",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_74fdd4d677294768bb63225cb4684b6d",
      "value": 1
     }
    },
    "af2a6221808f40b4813a158e3ae91559": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b49b0dd4ac514e6b80caa2932b5d222d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "b4a6bd56fccd47c88d23426c5da96930": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c2381c3d3724434c9e0299a5be8cbc22": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_daed6024f35146d9b0f1ad959e6f284e",
       "IPY_MODEL_c8201ec977bd42238b3575fcb1e37450",
       "IPY_MODEL_71c2b430b94b4639bc8468c202801cca"
      ],
      "layout": "IPY_MODEL_b4a6bd56fccd47c88d23426c5da96930"
     }
    },
    "c3ad5d149e0047628c3e0aefd29c7460": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "PasswordModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "PasswordModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "PasswordView",
      "continuous_update": true,
      "description": "Token:",
      "description_tooltip": null,
      "disabled": false,
      "layout": "IPY_MODEL_5a0871be46944d6d857bf3da82bd2f85",
      "placeholder": "​",
      "style": "IPY_MODEL_6ead92dda89645149a5e0a7e4060bcb7",
      "value": ""
     }
    },
    "c42b4b6b36944f34b0cd0027c90bc968": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c8201ec977bd42238b3575fcb1e37450": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "danger",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d2a4ecba9b364f91af3b5d13df9a7390",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_785502d8755f43cea3da690322f26d0d",
      "value": 0
     }
    },
    "cc05745cc3f6421f84af9a1f492bbea2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d2a4ecba9b364f91af3b5d13df9a7390": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d5b2c76f54ae4bb78985ad253373408d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d5b3dc1fa6c743e7b1e05a091c22b270": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_aa0798372ed149a18994104d7f9a3d97",
      "placeholder": "​",
      "style": "IPY_MODEL_3ab74c3f04634c16b24630f9b6d1798c",
      "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
     }
    },
    "d72eb7bedd624133ac08ccb510ce901a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1a5849c19ac44c5cb37de2bc04f8a8f6",
      "placeholder": "​",
      "style": "IPY_MODEL_a1e4b90f504e4c59ac795f8a3a6553d2",
      "value": " 1/1 [00:02&lt;00:00,  2.53s/it]"
     }
    },
    "da7804c13630441fbad5098822d99052": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ad67045339e340a0a831369fef5bcdcf",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_43d93313aaad4d7aa1ac513b8d69300c",
      "value": 1
     }
    },
    "daed6024f35146d9b0f1ad959e6f284e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4d8836ba902043358883ac3729ad621a",
      "placeholder": "​",
      "style": "IPY_MODEL_008ca4ebe46d48cb989e97aab9529604",
      "value": "  0%"
     }
    },
    "e0c423cb57e949f08255453100fd3e69": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e7d7757a758348f898a4d308695ebed7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e8c5889da2b847f6ac1512ab06bdd281": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ed568e897d1c4b7a9eb2c9e69997d29b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ButtonStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ButtonStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "button_color": null,
      "font_weight": ""
     }
    },
    "f01c225a559f443593a0989fa78fc6cb": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f4bdbfe4a65c428bbcbb95a9dc4c6356": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f919b355bcd344059b9161e6fac7cbac": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
